FP DNG data limit: perform calculations in 64 bit

Origin: https://github.com/LibRaw/LibRaw/commit/dae685a198309b978805f098bafe5d951dbc8747
Bug: https://talosintelligence.com/vulnerability_reports/TALOS-2026-2364
Bug-Debian: https://bugs.debian.org/1133845
Bug-Debian-Security: https://security-tracker.debian.org/tracker/CVE-2026-20884

Gbp-Pq: Topic CVE-2026-20884
Gbp-Pq: Name 02-dae685a19.patch

diff --git a/src/decoders/fp_dng.cpp b/src/decoders/fp_dng.cpp
index 6f82e896ca22b887bdb2fa6f6d0b9713ec51a7de..b18ca337f136802cf9570ddebb451b799143ae56 100644 (file)
--- a/src/decoders/fp_dng.cpp
+++ b/src/decoders/fp_dng.cpp
@@ -376,12 +376,15 @@ void LibRaw::deflate_dng_load_raw()
     break;
   }
 
-  unsigned tilePixels = tiles.tileWidth * tiles.tileHeight;
+  INT64 tilePixels =  INT64(tiles.tileWidth) * INT64(tiles.tileHeight);
   unsigned pixelSize = sizeof(float) * ifd->samples;
-  unsigned tileBytes = tilePixels * pixelSize;
-  unsigned tileRowBytes = tiles.tileWidth * pixelSize;
+  INT64 tileBytes = tilePixels * INT64(pixelSize);
+  INT64 tileRowBytes = INT64(tiles.tileWidth) * INT64(pixelSize);
 
-  if(INT64(tiles.maxBytesInTile) > INT64(imgdata.rawparams.max_raw_memory_mb) * INT64(1024 * 1024) )
+  if(INT64(tiles.maxBytesInTile) > INT64(imgdata.rawparams.max_raw_memory_mb) * 1024LL * 1024LL )
+    throw LIBRAW_EXCEPTION_TOOBIG;
+
+  if (tileBytes + tileRowBytes > INT64(imgdata.rawparams.max_raw_memory_mb) * 1024LL * 1024LL)
     throw LIBRAW_EXCEPTION_TOOBIG;
 
   std::vector<uchar> cBuffer(tiles.maxBytesInTile,0);